feat(protocol): serve schema blocks the advertisement cannot carry - #37

rg -n 'pull_request:|actions/checkout|github.head_ref|go test|go build' .github/workflows

set -e
echo "== workflow files =="
ls -la .github/workflows
echo "== go.yaml 1-60 =="
sed -n '1,60p' .github/workflows/go.yaml
echo "== python.yaml =="
sed -n '1,60p' .github/workflows/python.yaml
echo "== build.yaml =="
sed -n '1,60p' .github/workflows/build.yaml
echo "== triggers across workflows =="
rg -n -e 'merge_group' -e '^on:' -e 'push:' -e 'pull_request' -e 'workflow_call' .github/workflows || test "$?" -eq 1
echo "== git history of go.yaml checkout =="
git log --oneline -n 8 -- .github/workflows/go.yaml || true
git show 6f085a37fbcf7b65a05e19fe531b0950786ee1c1:.github/workflows/go.yaml 2>/dev/null | sed -n '1,45p' || echo "base go.yaml unavailable"

set -e
echo "== Dockerfile =="
test -f Dockerfile && sed -n '1,200p' Dockerfile || echo "no Dockerfile at root"
echo "== pre-commit config =="
test -f .pre-commit-config.yaml && sed -n '1,200p' .pre-commit-config.yaml || echo "no pre-commit config"
echo "== go references outside go.yaml =="
rg -n -F -e 'go test' -e 'go build' -e 'go vet' -e 'golang' -e 'xbot_service_interface_go' -- . --glob '!.git/**' || test "$?" -eq 1
echo "== Go module imports of C++ or other packages =="
rg -n -F -- 'xbot_service_interface_go' xbot_service_interface_go/go.mod || test "$?" -eq 1

rg -n '"(description|metadata|documentation|units|type|version)"|splitDescription|json_service.items' services codegen --glob '*.json' --glob '*.py' 2>/dev/null | head -100

set -e
echo "== schema =="
cat codegen/service.schema.json
echo "== codegen splitDescription/loadService/render =="
sed -n '80,175p' codegen/xbot_codegen/xbot_codegen.py
echo "== templates =="
rg -n -F -e 'omitted' -e 'service_cbor' -e 'service_json' codegen/templates
echo "== python merge/fetch =="
rg -n -F -e 'omitted' xbot_service_interface_py/xbot_service_interface
echo "== firmware receiver =="
rg -n -F -e 'omitted' -e 'DESCRIPTION_PART' -e 'HandleDescriptionRequest' libxbot-service/src libxbot-service/include || test "$?" -eq 1
echo "== service json keys in repo definitions =="
rg -n --glob '*.json' -e '"description"' -e '"units"' -e '"metadata"' . || test "$?" -eq 1

rg -n 'arg1|omitted_cbor|SERVICE_OMITTED_CBOR|pack_header' codegen/xbot_codegen/xbot_codegen.py codegen/templates libxbot-service/src/Service.cpp xbot_service_interface_py/xbot_service_interface/discovery.py

set -e
echo "== XbotHeader.hpp =="
nl -ba include/xbot/datatypes/XbotHeader.hpp | sed -n '1,80p'
echo "== DescriptionRequestPayload.hpp =="
nl -ba include/xbot/datatypes/DescriptionRequestPayload.hpp | sed -n '1,40p'
echo "== datatypes.py =="
nl -ba xbot_service_interface_py/xbot_service_interface/datatypes.py | sed -n '1,140p'
echo "== discovery.py fetch =="
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '160,260p'
echo "== Service.cpp omitted/arg handling and fillHeader =="
rg -n -F -e 'GetOmittedDescription' -e 'arg1' -e 'arg2' libxbot-service/src/Service.cpp || test "$?" -eq 1

set -e
echo "== callers of _omitted_blocks =="
rg -n -F -e '_omitted_blocks' -e 'FETCH_MAX_WINDOWS' -e 'FETCH_TIMEOUT_S' xbot_service_interface_py/ || test "$?" -eq 1
echo "== discovery.py 1-160 =="
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '1,160p'
echo "== discovery.py 286-320 =="
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '280,320p'
echo "== codegen splitDescription =="
nl -ba codegen/xbot_codegen/xbot_codegen.py | sed -n '95,150p'
echo "== max_packet_size =="
rg -n -F -e 'max_packet_size' include/ libxbot-service/ || test "$?" -eq 1

set -e
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '252,300p'
echo "== test windows =="
nl -ba xbot_service_interface_py/tests/test_description_fetch.py | sed -n '83,130p'

sed -n '90,150p' codegen/xbot_codegen/xbot_codegen.py
sed -n '317,370p' libxbot-service/src/Service.cpp
sed -n '167,247p' xbot_service_interface_py/xbot_service_interface/discovery.py

sed -n '30,70p' include/xbot/datatypes/XbotHeader.hpp
echo '---- uint8 search ----'
rg -n -F -e 'uint8_t' -e 'arg1' -- include/xbot/datatypes/XbotHeader.hpp || test "$?" -eq 1
echo '---- DescriptionRequestPayload ----'
sed -n '1,26p' include/xbot/datatypes/DescriptionRequestPayload.hpp
echo '---- Python pack functions ----'
rg -n -A14 -F -e 'def pack_header' -e 'def pack_description_request' -- xbot_service_interface_py/ || test "$?" -eq 1

echo '---- HEADER_FORMAT / constants ----'
rg -n -F -e 'HEADER_FORMAT' -e 'MAX_PACKET_SIZE' -e 'HEADER_SIZE' -- xbot_service_interface_py/xbot_service_interface/ || test "$?" -eq 1
echo '---- callers of _omitted_blocks ----'
rg -n -F -e '_omitted_blocks(' -- xbot_service_interface_py/xbot_service_interface/ || test "$?" -eq 1
echo '---- discovery.py caller context ----'
rg -n -A22 -F -e 'self._omitted_blocks(' -- xbot_service_interface_py/xbot_service_interface/discovery.py || test "$?" -eq 1

rg -n 'DESCRIPTION_REQUEST|HandleDescriptionRequest|transmitPacket\(' libxbot-service/src libxbot-service/include

set -e
nl -ba libxbot-service/src/Service.cpp | sed -n '240,372p'
echo '--- GetOmittedDescription / omitted'
rg -n -F -e 'GetOmittedDescription' -e 'SERVICE_OMITTED_CBOR' -e 'omitted' -- libxbot-service codegen || test "$?" -eq 1
echo '--- payload header'
nl -ba include/xbot/datatypes/DescriptionRequestPayload.hpp 2>/dev/null || find . -name 'DescriptionRequestPayload.hpp' -not -path '*/node_modules/*' -exec nl -ba {} \;
echo '--- rate limit / source filter search'
rg -n -i -e 'rate' -e 'throttl' -e 'ratelimit' -e 'source_ip' -e 'sender' -e 'from_ip' -e 'remote_ip' -- libxbot-service/src libxbot-service/include || test "$?" -eq 1
echo '--- socket receive'
nl -ba libxbot-service/src/portable/linux/socket.cpp | sed -n '1,200p' | rg -n -e 'recv' -e 'bind' -e 'INADDR' -e 'multicast' || test "$?" -eq 1

set -e
echo '--- XbotHeader'
rg -n -F -e 'struct XbotHeader' -- include libxbot-service || test "$?" -eq 1
f=$(rg -l -F 'struct XbotHeader' -- include libxbot-service | head -1 || true); [ -n "$f" ] && nl -ba "$f" | sed -n '1,80p'
echo '--- max_packet_size'
rg -n -F -e 'max_packet_size' -- include libxbot-service || test "$?" -eq 1
echo '--- HandleClaimMessage'
nl -ba libxbot-service/src/Service.cpp | sed -n '369,420p'
echo '--- socket initialize bind'
nl -ba libxbot-service/src/portable/linux/socket.cpp | sed -n '69,130p'
echo '--- trust / security docs'
rg -n -i -e 'trusted' -e 'untrusted' -e 'authent' -e 'amplif' -e 'spoof' -e 'firewall' -- Readme.md codegen/README.md xbot_service_interface_py/README.md libxbot-service || test "$?" -eq 1

rg -n -F -e '_omitted_blocks' -e 'def _on_advert' -e 'def _handle' -e 'except' -- xbot_service_interface_py/xbot_service_interface/discovery.py || test "$?" -eq 1
echo ---
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '160,260p'

nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '252,340p'

sed -n '130,165p' xbot_service_interface_py/xbot_service_interface/discovery.py; sed -n '250,336p' xbot_service_interface_py/xbot_service_interface/discovery.py

rg -n -F -e 'FETCH_TIMEOUT_S' -e 'FETCH_MAX_WINDOWS' -e '_handle_packet' -e '_recv_loop' -e '_omitted_cache' -e 'threading.Thread' -e 'ThreadPool' -e 'executor' -e 'Thread(' -- xbot_service_interface_py/ || test "$?" -eq 1
echo '---discovery.py head---'
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '1,100p'
echo '---discovery.py rest---'
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '160,200p'
echo '---other discovery users---'
rg -n -F -e 'ServiceDiscovery' -e 'on_service_found' -- xbot_service_interface_py/ || test "$?" -eq 1
echo '---advertise period---'
rg -n -F -e 'SERVICE_ADVERTISEMENT' -e 'advertis' -- libxbot-service/src/ libxbot-service/include/ || test "$?" -eq 1

sed -n '155,265p' xbot_service_interface_py/xbot_service_interface/discovery.py

rg -n -F -e '_omitted_blocks' -e '_handle_packet' -e 'FETCH_TIMEOUT_S' -e 'FETCH_MAX_WINDOWS' -e '_recv_loop' -e 'Thread(' -e 'threading' -- xbot_service_interface_py/xbot_service_interface/discovery.py || test "$?" -eq 1
echo '---'
nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '1,155p'

nl -ba xbot_service_interface_py/xbot_service_interface/discovery.py | sed -n '250,330p'

sed -n '170,247p' xbot_service_interface_py/xbot_service_interface/discovery.py; sed -n '267,336p' xbot_service_interface_py/xbot_service_interface/discovery.py

set -u
echo "== header helpers / struct (datatypes.py) =="
rg -n -F -e 'def pack_header' -e 'def unpack_header' -e 'HEADER_SIZE' -e 'struct.Struct' -e 'struct.pack' -e 'struct.unpack' -- xbot_service_interface_py/xbot_service_interface/ || test "$?" -eq 1
echo "== discovery.py: constants, imports, fetch helpers, sid usage =="
rg -n -F -e 'FETCH_' -e '_route_source_ip' -e '_fetch_omitted' -e '_omitted_cache' -e 'sid' -e 'pack_description_request' -e 'MAX_PACKET_SIZE' -- xbot_service_interface_py/xbot_service_interface/discovery.py || test "$?" -eq 1
echo "== ServiceSchema.from_dict =="
rg -n -F -e 'def from_dict' -e 'class ServiceSchema' -e 'omitted' -- xbot_service_interface_py/xbot_service_interface/datatypes.py || test "$?" -eq 1
echo "== service-side DESCRIPTION_PART send path =="
rg -n -F -e 'DESCRIPTION_PART' -e 'DescriptionRequestPayload' -e 'SERVICE_OMITTED_CBOR' -- libxbot-service/src/Service.cpp include/xbot/datatypes/XbotHeader.hpp include/xbot/datatypes/DescriptionRequestPayload.hpp || test "$?" -eq 1
echo "== tests touching sender/sid/spoof =="
rg -n -i -e 'spoof' -e 'sender' -e 'source' -e 'foreign' -e 'sid' -- xbot_service_interface_py/tests/test_description_fetch.py || test "$?" -eq 1

set -u
echo "== datatypes.py header layout and helpers =="
sed -n '15,40p;84,110p' xbot_service_interface_py/xbot_service_interface/datatypes.py
echo "== locate ServiceSchema and from_dict =="
rg -n -F -e 'class ServiceSchema' -e 'def from_dict' -- xbot_service_interface_py/ || test "$?" -eq 1
echo "== Service.cpp DESCRIPTION_PART reply path =="
sed -n '314,371p' libxbot-service/src/Service.cpp

set -u
echo "== schema.py ServiceSchema =="
sed -n '1,80p' xbot_service_interface_py/xbot_service_interface/schema.py
echo "== Service.cpp fillHeader =="
rg -n -F -e 'fillHeader' -- libxbot-service/src/Service.cpp || test "$?" -eq 1
echo "== discovery.py receive loop source handling =="
sed -n '250,266p' xbot_service_interface_py/xbot_service_interface/discovery.py

set -u
sed -n '189,205p' libxbot-service/src/Service.cpp

sed -n '299,336p' xbot_service_interface_py/xbot_service_interface/discovery.py; rg -n 'on_service_found|_on_service_discovered' xbot_service_interface_py/xbot_service_interface

set -e
echo '=== discovery.py 240-345 ==='
sed -n '240,345p' xbot_service_interface_py/xbot_service_interface/discovery.py | cat -n | sed 's/^/L+239 /'
echo '=== discovery.py: _omitted_blocks and _fetch_omitted ==='
rg -n -F -e '_omitted_blocks' -e '_fetch_omitted' -e 'def ' xbot_service_interface_py/xbot_service_interface/discovery.py || test "$?" -eq 1
echo '=== manager.py 90-175 ==='
sed -n '90,175p' xbot_service_interface_py/xbot_service_interface/manager.py
echo '=== interface.py 420-470 ==='
sed -n '420,470p' xbot_service_interface_py/xbot_service_interface/interface.py

set -e
echo '=== discovery.py 104-252 ==='
sed -n '104,252p' xbot_service_interface_py/xbot_service_interface/discovery.py | cat -n | sed 's/^/L+103 /'
echo '=== tests: rediscovery / version / endpoint ==='
rg -n -i -e 'rediscover' -e 'version' -e 'endpoint' -e 'same' -e 'cache' -- xbot_service_interface_py/tests/test_description_fetch.py || test "$?" -eq 1
echo '=== drop_service / _disconnected call sites ==='
rg -n -F -e 'drop_service' -e 'get_service_info' -e '_on_disconnected' -- xbot_service_interface_py/xbot_service_interface || test "$?" -eq 1

set -e
echo '=== datatypes.py: ServiceSchema.from_dict ==='
rg -n -F -e 'class ServiceSchema' -e 'def from_dict' -- xbot_service_interface_py/xbot_service_interface/datatypes.py || test "$?" -eq 1
echo '=== interface.py: constructor schema default ==='
rg -n -F -e 'def __init__' -e 'schema' -- xbot_service_interface_py/xbot_service_interface/interface.py | head -n 40 || test "$?" -eq 1
echo '=== codegen: omitted partition ==='
rg -n -F -e 'omitted' -e 'OMITTED' -- codegen/xbot_codegen/xbot_codegen.py || test "$?" -eq 1
echo '=== tests: fetch failure ==='
rg -n -F -e 'def test_' -- xbot_service_interface_py/tests/test_description_fetch.py || test "$?" -eq 1

set -e
echo '=== schema.py: from_dict / is_compatible ==='
rg -n -F -e 'def from_dict' -e 'def is_compatible' -e 'class ServiceSchema' -- xbot_service_interface_py/xbot_service_interface/schema.py || test "$?" -eq 1
echo '=== test_description_fetch.py 99-175 ==='
sed -n '99,175p' xbot_service_interface_py/tests/test_description_fetch.py
echo '=== xbot_codegen.py 95-125 ==='
sed -n '95,125p' codegen/xbot_codegen/xbot_codegen.py

set -e
sed -n '1,60p' xbot_service_interface_py/xbot_service_interface/schema.py | cat -n